Add "pull --localcache-repo"

This is a lot like `git clone --reference`, but we chose "localcache" as the
term "reference" is already used.

The main use case I'm targeting this for is the Fedora Atomic Host installer
case where we embed the repo content in the installer, but we may want to
kickstart and download newer content. There, while we want to get a newer ref,
we can still use the local repo as an object cache, since we have it sitting
there in memory anyways.

Another case is where one has a host ostree (say e.g. Fedora Atomic
Workstation), and one wants to create a local archive mirror of FAH. Then one
can use `pull --reference /ostree/repo` and pull the common objects (e.g.
contents of `bash.rpm` etc.)

Closes: https://github.com/ostreedev/ostree/issues/975
Closes: #982
Approved by: jlebon

diff --git a/Makefile-tests.am b/Makefile-tests.am
index 348a8e4a0f63a99657ebb21717756f1f9c38fa63..444b6636f674d17294d19281fb557d6cd452785b 100644 (file)
--- a/Makefile-tests.am
+++ b/Makefile-tests.am
@@ -78,6 +78,7 @@ _installed_or_uninstalled_test_scripts = \
        tests/test-pull-repeated.sh \
        tests/test-pull-untrusted.sh \
        tests/test-pull-override-url.sh \
+       tests/test-pull-localcache.sh \
        tests/test-local-pull.sh \
        tests/test-local-pull-depth.sh \
        tests/test-gpg-signed-commit.sh \

diff --git a/man/ostree-pull.xml b/man/ostree-pull.xml
index 24ab0b72cf5d11a107e5c7e443edcfe2feab0aa4..394a29c27da9689ee5f5341e1d9b9708a526ad13 100644 (file)
--- a/man/ostree-pull.xml
+++ b/man/ostree-pull.xml
@@ -73,6 +73,16 @@ Boston, MA 02111-1307, USA.
                 </para></listitem>
             </varlistentry>
 
+            <varlistentry>
+              <term><option>--localcache-repo</option></term>
+
+              <listitem><para>
+                Like git's <literal>clone --reference</literal>.  Reuse the provided
+                OSTree repo as a local object cache of objects when doing HTTP fetches.
+                May be specified multiple times.
+              </para></listitem>
+            </varlistentry>
+
             <varlistentry>
                 <term><option>--untrusted</option></term>
 

diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index fb0c9e824b96fd57966c8cd677b458acd51965a4..7346b299003dd2aec1ce99e0ec81254562d6ed8f 100644 (file)
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -64,6 +64,7 @@ typedef struct {
   GPtrArray     *meta_mirrorlist;    /* List of base URIs for fetching metadata */
   GPtrArray     *content_mirrorlist; /* List of base URIs for fetching content */
   OstreeRepo   *remote_repo_local;
+  GPtrArray    *localcache_repos; /* Array<OstreeRepo> */
 
   GMainContext    *main_context;
   GCancellable *cancellable;
@@ -118,6 +119,9 @@ typedef struct {
   guint             n_fetched_deltapart_fallbacks;
   guint             n_fetched_metadata;
   guint             n_fetched_content;
+  /* Objects from pull --localcache-repo */
+  guint             n_fetched_localcache_metadata;
+  guint             n_fetched_localcache_content;
 
   int               maxdepth;
   guint64           start_time;
@@ -236,6 +240,8 @@ update_progress (gpointer user_data)
                              "scanned-metadata", "u", n_scanned_metadata,
                              "bytes-transferred", "t", bytes_transferred,
                              "start-time", "t", start_time,
+                             "metadata-fetched-localcache", "u", pull_data->n_fetched_localcache_metadata,
+                             "content-fetched-localcache", "u", pull_data->n_fetched_localcache_content,
                              /* Deltas */
                              "fetched-delta-parts",
                                   "u", pull_data->n_fetched_deltaparts,
@@ -606,16 +612,15 @@ validate_bareuseronly_mode (OtPullData *pull_data,
  */
 static gboolean
 import_one_local_content_object (OtPullData *pull_data,
+                                 OstreeRepo *src_repo,
                                  const char *checksum,
                                  GCancellable *cancellable,
                                  GError    **error)
 {
-  g_assert (pull_data->remote_repo_local);
-
   const gboolean trusted = !pull_data->is_untrusted;
   if (trusted && !pull_data->is_bareuseronly_files)
     {
-      if (!ostree_repo_import_object_from_with_trust (pull_data->repo, pull_data->remote_repo_local,
+      if (!ostree_repo_import_object_from_with_trust (pull_data->repo, src_repo,
                                                       OSTREE_OBJECT_TYPE_FILE, checksum,
                                                       trusted,
                                                       cancellable, error))
@@ -630,7 +635,7 @@ import_one_local_content_object (OtPullData *pull_data,
       g_autoptr(GFileInfo) content_finfo = NULL;
       g_autoptr(GVariant) content_xattrs = NULL;
 
-      if (!ostree_repo_load_file (pull_data->remote_repo_local, checksum,
+      if (!ostree_repo_load_file (src_repo, checksum,
                                   &content_input, &content_finfo, &content_xattrs,
                                   cancellable, error))
         return FALSE;
@@ -709,16 +714,43 @@ scan_dirtree_object (OtPullData   *pull_data,
       /* Is this a local repo? */
       if (pull_data->remote_repo_local)
         {
-          if (!import_one_local_content_object (pull_data, file_checksum, cancellable, error))
+          if (!import_one_local_content_object (pull_data, pull_data->remote_repo_local,
+                                                file_checksum, cancellable, error))
             return FALSE;
+
+          /* Note early loop continue */
+          continue;
         }
-      else
+
+      /* We're doing HTTP, but see if we have the object in a local cache first */
+      gboolean did_import_from_cache_repo = FALSE;
+      if (pull_data->localcache_repos)
         {
-          /* In this case we're doing HTTP pulls */
-          g_hash_table_add (pull_data->requested_content, file_checksum);
-          enqueue_one_object_request (pull_data, file_checksum, OSTREE_OBJECT_TYPE_FILE, path, FALSE, FALSE);
-          file_checksum = NULL;  /* Transfer ownership */
+          for (guint j = 0; j < pull_data->localcache_repos->len; j++)
+            {
+              OstreeRepo *localcache_repo = pull_data->localcache_repos->pdata[j];
+              gboolean localcache_repo_has_obj;
+
+              if (!ostree_repo_has_object (localcache_repo, OSTREE_OBJECT_TYPE_FILE, file_checksum,
+                                           &localcache_repo_has_obj, cancellable, error))
+                return FALSE;
+              if (!localcache_repo_has_obj)
+                continue;
+              if (!import_one_local_content_object (pull_data, localcache_repo, file_checksum,
+                                                    cancellable, error))
+                return FALSE;
+              did_import_from_cache_repo = TRUE;
+              pull_data->n_fetched_localcache_content++;
+              break;
+            }
         }
+      if (did_import_from_cache_repo)
+        continue; /* Note early continue */
+
+      /* Not available locally, queue a HTTP request */
+      g_hash_table_add (pull_data->requested_content, file_checksum);
+      enqueue_one_object_request (pull_data, file_checksum, OSTREE_OBJECT_TYPE_FILE, path, FALSE, FALSE);
+      file_checksum = NULL;  /* Transfer ownership */
     }
 
   g_autoptr(GVariant) dirs_variant = g_variant_get_child_value (tree, 1);
@@ -1527,6 +1559,33 @@ scan_one_metadata_object_c (OtPullData         *pull_data,
       is_stored = TRUE;
       is_requested = TRUE;
     }
+  /* Do we have any localcache repos? */
+  else if (!is_stored && pull_data->localcache_repos)
+    {
+      for (guint i = 0; i < pull_data->localcache_repos->len; i++)
+        {
+          OstreeRepo *refd_repo = pull_data->localcache_repos->pdata[i];
+          gboolean localcache_repo_has_obj;
+
+          if (!ostree_repo_has_object (refd_repo, objtype, tmp_checksum,
+                                       &localcache_repo_has_obj, cancellable, error))
+            return FALSE;
+          if (!localcache_repo_has_obj)
+            continue;
+          if (!ostree_repo_import_object_from_with_trust (pull_data->repo, refd_repo,
+                                                          objtype, tmp_checksum,
+                                                          !pull_data->is_untrusted,
+                                                          cancellable, error))
+            return FALSE;
+          /* See comment above */
+          if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
+            g_hash_table_add (pull_data->fetched_detached_metadata, g_strdup (tmp_checksum));
+          is_stored = TRUE;
+          is_requested = TRUE;
+          pull_data->n_fetched_localcache_metadata++;
+          break;
+        }
+    }
 
   if (!is_stored && !is_requested)
     {
@@ -2865,6 +2924,7 @@ initiate_request (OtPullData                 *pull_data,
  *   * inherit-transaction (b): Don't initiate, finish or abort a transaction, useful to do multiple pulls in one transaction.
  *   * http-headers (a(ss)): Additional headers to add to all HTTP requests
  *   * update-frequency (u): Frequency to call the async progress callback in milliseconds, if any; only values higher than 0 are valid
+ *   * localcache-repos (as): File paths for local repos to use as caches when doing remote fetches
  */
 gboolean
 ostree_repo_pull_with_options (OstreeRepo             *self,
@@ -2903,6 +2963,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   gboolean inherit_transaction = FALSE;
   g_autoptr(GHashTable) updated_requested_refs_to_fetch = NULL;  /* (element-type OstreeCollectionRef utf8) */
   int i;
+  g_autofree char **opt_localcache_repos = NULL;
 
   if (options)
     {
@@ -2929,6 +2990,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
       (void) g_variant_lookup (options, "inherit-transaction", "b", &inherit_transaction);
       (void) g_variant_lookup (options, "http-headers", "@a(ss)", &pull_data->extra_headers);
       (void) g_variant_lookup (options, "update-frequency", "u", &update_frequency);
+      (void) g_variant_lookup (options, "localcache-repos", "^a&s", &opt_localcache_repos);
     }
 
   g_return_val_if_fail (pull_data->maxdepth >= -1, FALSE);
@@ -2993,6 +3055,20 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
                                                              (GDestroyNotify)fetch_object_data_free);
   pull_data->pending_fetch_deltaparts = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)fetch_static_delta_data_free, NULL);
 
+  if (opt_localcache_repos && *opt_localcache_repos)
+    {
+      pull_data->localcache_repos = g_ptr_array_new_with_free_func (g_object_unref);
+      for (char **it = opt_localcache_repos; it && *it; it++)
+        {
+          const char *localcache_path = *it;
+          g_autoptr(GFile) localcache_file = g_file_new_for_path (localcache_path);
+          g_autoptr(OstreeRepo) cacherepo = ostree_repo_new (localcache_file);
+          if (!ostree_repo_open (cacherepo, cancellable, error))
+            goto out;
+          g_ptr_array_add (pull_data->localcache_repos, g_steal_pointer (&cacherepo));
+        }
+    }
+
   if (dir_to_pull != NULL || dirs_to_pull != NULL)
     {
       pull_data->dirs = g_ptr_array_new_with_free_func (g_free);
@@ -3722,6 +3798,11 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
       else
         g_string_append_printf (buf, "%u metadata, %u content objects fetched",
                                 pull_data->n_fetched_metadata, pull_data->n_fetched_content);
+      if (pull_data->n_fetched_localcache_metadata ||
+          pull_data->n_fetched_localcache_content)
+        g_string_append_printf (buf, " (%u meta, %u content local)",
+                                pull_data->n_fetched_localcache_metadata,
+                                pull_data->n_fetched_localcache_content);
 
       g_string_append_printf (buf, "; %" G_GUINT64_FORMAT " %s transferred in %u seconds",
                               (guint64)(bytes_transferred / shift),
@@ -3769,6 +3850,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   g_clear_object (&pull_data->fetcher);
   g_clear_pointer (&pull_data->extra_headers, (GDestroyNotify)g_variant_unref);
   g_clear_object (&pull_data->cancellable);
+  g_clear_pointer (&pull_data->localcache_repos, (GDestroyNotify)g_ptr_array_unref);
   g_clear_object (&pull_data->remote_repo_local);
   g_free (pull_data->remote_name);
   g_clear_pointer (&pull_data->meta_mirrorlist, (GDestroyNotify) g_ptr_array_unref);

diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c
index edb0955088ac9fbd1ba3ffb3ccf147f3326f16c6..7898e107c42110944207049344eb803734f2c692 100644 (file)
--- a/src/ostree/ot-builtin-pull.c
+++ b/src/ostree/ot-builtin-pull.c
@@ -41,6 +41,7 @@ static char* opt_cache_dir;
 static int opt_depth = 0;
 static int opt_frequency = 0;
 static char* opt_url;
+static char** opt_localcache_repos;
 
 static GOptionEntry options[] = {
    { "commit-metadata-only", 0, 0, G_OPTION_ARG_NONE, &opt_commit_only, "Fetch only the commit metadata", NULL },
@@ -57,6 +58,7 @@ static GOptionEntry options[] = {
    { "url", 0, 0, G_OPTION_ARG_STRING, &opt_url, "Pull objects from this URL instead of the one from the remote config", NULL },
    { "http-header", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_http_headers, "Add NAME=VALUE as HTTP header to all requests", "NAME=VALUE" },
    { "update-frequency", 0, 0, G_OPTION_ARG_INT, &opt_frequency, "Sets the update frequency, in milliseconds (0=1000ms) (default: 0)", "FREQUENCY" },
+   { "localcache-repo", 'L', 0, G_OPTION_ARG_FILENAME_ARRAY, &opt_localcache_repos, "Add REPO as local cache source for objects during this pull", "REPO" },
    { NULL }
  };
 
@@ -281,6 +283,9 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **
     if (override_commit_ids)
       g_variant_builder_add (&builder, "{s@v}", "override-commit-ids",
                              g_variant_new_variant (g_variant_new_strv ((const char*const*)override_commit_ids->pdata, override_commit_ids->len)));
+    if (opt_localcache_repos)
+      g_variant_builder_add (&builder, "{s@v}", "localcache-repos",
+                             g_variant_new_variant (g_variant_new_strv ((const char*const*)opt_localcache_repos, -1)));
 
     if (opt_http_headers)
       {

diff --git a/tests/test-pull-localcache.sh b/tests/test-pull-localcache.sh
new file mode 100755 (executable)
index 0000000..5f810ac
--- /dev/null
+++ b/tests/test-pull-localcache.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+#
+# Copyright (C) 2017 Red Hat, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -euo pipefail
+
+. $(dirname $0)/libtest.sh
+
+setup_fake_remote_repo1 "archive"
+
+echo '1..1'
+
+cd ${test_tmpdir}
+gnomerepo_url="$(cat httpd-address)/ostree/gnomerepo"
+ostree_repo_init repo --mode "archive"
+ostree_repo_init repo-local --mode "archive"
+for repo in repo{,-local}; do
+    ${CMD_PREFIX} ostree --repo=${repo} remote add --set=gpg-verify=false origin ${gnomerepo_url}
+done
+
+# Pull the contents to our local cache
+${CMD_PREFIX} ostree --repo=repo-local pull origin main
+rm files -rf
+${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo checkout main files
+echo anewfile > files/anewfile
+${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit -b main --tree=dir=files
+
+${CMD_PREFIX} ostree --repo=repo pull -L repo-local origin main >out.txt
+assert_file_has_content out.txt '3 metadata, 1 content objects fetched (4 meta, 5 content local)'
+echo "ok pull --reference"